package ru.fadeev.crammer.phrasedb;

import java.io.*;
import java.sql.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.*;


public class PhraseDBFiller {
	private String url;
	private String username;
	private String password;
	
	public PhraseDBFiller() throws IOException{
		Properties props = new Properties();
        props.load(this.getClass().getClassLoader().getResourceAsStream("database.properties"));
        String host = props.getProperty("host");
        String port = props.getProperty("port");
        String dbname = props.getProperty("dbname");
        username = props.getProperty("username");
        password = props.getProperty("password");
        url = "jdbc:postgresql://" + host + ":" + port + "/" + dbname;          
	}
	
	public Connection getConnection() throws Exception {
		Class.forName("org.postgresql.Driver");
	    //String url = "jdbc:hsqldb:hsql://localhost/crammer";
		//String url ="jdbc:postgresql://localhost:5432/template1";		
	    //return DriverManager.getConnection(url, username, password);	
		return DriverManager.getConnection(url, username, password);	
	}
		
	public void checkDatabaseConsistency() throws Exception {
        Connection connection = getConnection();      

        Statement statement = connection.createStatement();
        ResultSet rs = statement.executeQuery(
        		"SELECT * FROM PG_TABLES WHERE TABLENAME='phrase'");
        
        if(!rs.next()) {
        	      	
        	statement.executeUpdate(
        			"CREATE TABLE PHRASE (" +
        				"ID SERIAL, " +
        				"PHRASE_TEXT VARCHAR(200), " +
        				"PHRASE_TSVECTOR TSVECTOR " +
        			")"
        			);	
        	
        	statement.executeUpdate(
        			"CREATE INDEX PHRASE_INDEX ON PHRASE USING GIN(" +
        				"PHRASE_TSVECTOR" +	        				
        			")"
        			);	
        	System.out.println("PHRASE table created");
        	
        }
        statement.close();
        connection.close();
	}
	
    public int insertDirectoryToDB(File directory) {   	
   	    TextFileFilter fileFilter = new TextFileFilter();
   	    
   		File[] files = directory.listFiles(fileFilter);
   		int phrasesCount = 0;
   		for (int i = 0; i < files.length; i++) {
   			if(files[i].isDirectory()) {
   				phrasesCount += insertDirectoryToDB(files[i]);
   			} else {
   				phrasesCount += insertFileToDB(files[i]);
   			}
   		}    	
   		return phrasesCount;
    }
    
    public int insertFileToDB(File file) {
		
    	/* Files longer than 100Mb are not being processed */
    	if(file.length() > 100000000) {
    		return 0;
    	}
    	
    	try {   	
    		RandomAccessFile randomAccess = new RandomAccessFile(file,"r");
    		byte[] array = new byte[(int)file.length()];

    		randomAccess.read(array);
    		
    		String content = new String(array); 
    		System.out.print("Adding file \"" + file.getName() + "\"... ");
    		int phraseCount = insertTextToDB(new String(content));
    		System.out.println("added (" + phraseCount + " phrases)");
    		randomAccess.close();
    		return phraseCount;
    		
    	} catch (Exception e) {
    		e.printStackTrace();
    		return 0;
    	}		
    }
     
	public String getCombinedDelimiter() {
		//String result = "[";
		//if(Settings.getBoolean("addPhrasePoint")) {
		//	result += "\\."; 
		//}
		//if(Settings.getBoolean("addPhraseExclamation")) {
		//	result += "\\!"; 
		//}
		//if(Settings.getBoolean("addPhraseQuestion")) {
		//	result += "\\?"; 
		//}
		//if(Settings.getBoolean("addPhraseLF")) {
		//	result += "\\n"; 
		//}
		//if(!Settings.getString("addPhraseCustomDelimiter").equals("")) {
		//	result += Settings.getString("addPhraseCustomDelimiter"); 
		//}
		//result += "]";
		//return result;
		return "[\\.\\!\\?]";
	}
	
    public int insertTextToDB(String text) { 
    	
    	int phrasesCount = 0;
    	String currentPhrase;
    	String delimiter = getCombinedDelimiter();
    	
    	/* Constructing delimiter */ 	
    	delimiter += "|(\\r\\n\\r\\n)";   	
    	int currentIndex = 0;
    	//if(Settings.getBoolean("addPhrasePreprocessing")) {
    		text = prepareText(text);
    	//}
    	
    	try {
	    	Connection connection = getConnection();  
	    	PreparedStatement statement = connection.prepareStatement(
	    			"INSERT INTO PHRASE " +
	    			"(PHRASE_TEXT, PHRASE_TSVECTOR) " +
	    			"VALUES (?, to_tsvector('english', ?))");
	    	
	    	//CallableStatement identityStatement = connection.prepareCall("{?= CALL IDENTITY()}");
	    	
			Pattern p = Pattern.compile(delimiter);
			Matcher matcher = p.matcher(text);
			while(matcher.find()) {
				
				/* Check for points that don't end sentence */
				//if(Settings.getBoolean("addPhrasePoint")) {
					currentPhrase = text.substring(currentIndex, matcher.start()+1);

					if(currentPhrase.length() >= 5) {
						String subPhrase = currentPhrase.substring(currentPhrase.length()-5, 
								currentPhrase.length());
						String exceptionList = "(\\WMrs\\.)|(\\WEsq\\.)";
							
							if(subPhrase.matches(exceptionList)) {
								continue;
							}
					} 
					if(currentPhrase.length() >= 4) {
						String subPhrase = currentPhrase.substring(currentPhrase.length()-4, 
								currentPhrase.length());
						String exceptionList = "(\\WMr\\.)|(\\WMs\\.)|(\\WSt\\.)|"+
								"(\\WPh\\.)|(\\WJr\\.)";
							
						if(subPhrase.matches(exceptionList)) {
							continue;
						}
					}
					if(currentPhrase.length() >= 3) {
						String subPhrase = currentPhrase.substring(currentPhrase.length()-3, 
								currentPhrase.length());
						String exceptionList = "(\\W[A-Z]\\.)|(\\Wi\\.)|(\\We\\.)";
						if(subPhrase.matches(exceptionList)) {
							continue;
						}
					}	
					if(currentPhrase.length() == 2) {
						String exceptionList = "([A-Z]\\.)";
						if(currentPhrase.matches(exceptionList)) {
							continue;
						}
					}	
				//}
				currentPhrase = text.substring(currentIndex, matcher.start());
				currentIndex = matcher.end();
				if (addPhraseToDB(currentPhrase, statement)) {
					//ResultSet resultSet = identityStatement.executeQuery();	
					//resultSet.next();
					//int id = resultSet.getInt(1);		
					phrasesCount++;
				}
			}
			currentPhrase = text.substring(currentIndex, text.length());
			if (addPhraseToDB(currentPhrase, statement)) {
			//	ResultSet resultSet = identityStatement.executeQuery();	
			//	resultSet.next();
				//int id = resultSet.getInt(1);
				phrasesCount++;
			}
			
			//identityStatement.close();
			statement.close();
			connection.close();
    	} catch (Exception e) {
	        e.printStackTrace();
	    }			
    	return phrasesCount;
    }
        
    public String prepareText(String text){
       	/* 
    	 * if there is single line arounded by blank lines 
    	 * and not ended by '.' (it means it's title and not processed) 
    	 */
    	String blankLine = "(\\r\\n(\\s)*\\r\\n)|(\\n(\\s)*\\n)";
    	String result="";
    	int currentIndex = 0;
		Pattern blankLinePattern = Pattern.compile(blankLine);
		Pattern lineFeedPattern = Pattern.compile("[\\r\\n]");
		Matcher blankLineMatcher = blankLinePattern.matcher(text);

		//long startTime = System.currentTimeMillis();
		while(blankLineMatcher.find()) {					
			String currentPhrase = text.substring(currentIndex, blankLineMatcher.start());
			Matcher lineFeedMatcher = lineFeedPattern.matcher(currentPhrase);
			if(lineFeedMatcher.find(0)) {
				result += text.substring(currentIndex, blankLineMatcher.end());
			}else if (currentPhrase.endsWith(".")){
				result += text.substring(currentIndex, blankLineMatcher.end());
			}
			currentIndex = blankLineMatcher.end();
		}
		//long processingTime = System.currentTimeMillis() - startTime; 
		result += text.substring(currentIndex, text.length());
    	String result2="";
    	currentIndex = 0;
		Pattern openBracketPattern = Pattern.compile("\\(");
		Pattern closeBracketPattern = Pattern.compile("\\)");
		Matcher openBracketMatcher = openBracketPattern.matcher(result);
		Matcher closeBracketMatcher = closeBracketPattern.matcher(result);
		while(openBracketMatcher.find(currentIndex)) {
			result2 += result.substring(currentIndex, openBracketMatcher.start());			
			if(closeBracketMatcher.find(openBracketMatcher.end())) {
				currentIndex = closeBracketMatcher.end();
			} else {
				currentIndex = openBracketMatcher.end();
			}
		}
		result2 += result.substring(currentIndex, result.length());		
		return result2;
    }
    
    public boolean addPhraseToDB(String phrase, PreparedStatement statement) {
    	int wordsCount = 0;
    	
		Pattern p = Pattern.compile("[\\w]+");
		Matcher matcher = p.matcher(phrase);
		while(matcher.find()) {
			wordsCount++;
		}
		//if (wordsCount < Settings.getInt("addPhraseMinLength")) {
		//	return false;
		//}
		if(wordsCount < 5) {
			return false;
		}
		if(phrase.trim().length() > 160) {
			return false;
		}		
			
		try {
			String compiledPhrase = phrase.trim().replaceAll("\\s+", " ");
			
			compiledPhrase = compiledPhrase.replaceAll("\\A[\"\\s]+", "");
			compiledPhrase = compiledPhrase.replaceAll("[\"\\s]+\\z", "");
			
			Pattern controlCharsPattern = Pattern.compile("[^\\u0020-\\u007E]");
			Matcher controlCharsMatcher = controlCharsPattern.matcher(compiledPhrase);
			if(controlCharsMatcher.find()) {
				return false;
			}
			statement.setString(1, compiledPhrase);
			statement.setString(2, compiledPhrase);
			statement.executeUpdate();
			return true;
		} catch (Exception e) {
	        e.printStackTrace();
	        return false;
	    }						
    }
    
    /**
     * Removes from texts titles and areas surrounded by brackets. 
     * @param text
     * @return
     */
    public String preprocessText(String text){
       	 
       	// Removing titles.
    	// If there is single(or double) line surrounded by blank lines 
    	// and not ended by '.'(or ?,!,") (it means it's title and not processed)
    	String blankLine = "(\\r\\n(\\s)*\\r\\n)|(\\n(\\s)*\\n)|(\\r(\\s)*\\r)";    	
    	String result="";
    	int currentIndex = 0;
    	Pattern blankLinePattern = Pattern.compile(blankLine, Pattern.UNIX_LINES);
		Pattern lineFeedPattern = Pattern.compile("[\\r\\n]");
		Matcher blankLineMatcher = blankLinePattern.matcher(text);

		//long startTime = System.currentTimeMillis();
		while(blankLineMatcher.find()) {					
			String currentPhrase = text.substring(currentIndex, blankLineMatcher.start());
			Matcher lineFeedMatcher = lineFeedPattern.matcher(currentPhrase);
			boolean isTitle = true;
			
			//If between blank lines more than one line
			if(lineFeedMatcher.find(0)) {
				String currentLine = currentPhrase.substring(0, lineFeedMatcher.start());
				//If between blank lines more than two lines
				if(lineFeedMatcher.find()) {
					isTitle = false;	
				} else if(currentLine.endsWith(".") || currentPhrase.endsWith(".")
						|| currentLine.endsWith("!") || currentPhrase.endsWith("!")
						|| currentLine.endsWith("?") || currentPhrase.endsWith("?")
						|| currentLine.endsWith("\"") || currentPhrase.endsWith("\"")
						|| currentLine.endsWith("'") || currentPhrase.endsWith("'")) {
					isTitle = false;
				}				
			}else if (currentPhrase.endsWith(".")
					|| currentPhrase.endsWith("!")
					|| currentPhrase.endsWith("?")
					|| currentPhrase.endsWith("\"")
					|| currentPhrase.endsWith("'")){
				isTitle = false;
			}
			if(!isTitle) {
				result += text.substring(currentIndex, blankLineMatcher.end());
			}
			currentIndex = blankLineMatcher.end();
		}
		
		//long processingTime = System.currentTimeMillis() - startTime; 
		result += text.substring(currentIndex, text.length());
		
		// Removing text surrounded by brackets.
    	String result2="";
    	currentIndex = 0;
		Pattern openBracketPattern = Pattern.compile("\\(");
		Pattern closeBracketPattern = Pattern.compile("\\)");
		Matcher openBracketMatcher = openBracketPattern.matcher(result);
		Matcher closeBracketMatcher = closeBracketPattern.matcher(result);
		while(openBracketMatcher.find(currentIndex)) {
			result2 += result.substring(currentIndex, openBracketMatcher.start());			
			if(closeBracketMatcher.find(openBracketMatcher.end())) {
				currentIndex = closeBracketMatcher.end();
			} else {
				currentIndex = openBracketMatcher.end();
			}
		}
		result2 += result.substring(currentIndex, result.length());		
		return result2;
    }
    
    public String postprocessText(String text) {
    	String result = text.trim().replaceAll("\\s+", " ");
    	return result;
    }
    
    public Vector<String> getPhrases(String _text) {
    	Vector<String> result = new Vector<String>();
    	String text = preprocessText(_text);
    	String sentenceDelimiter = "\\.";
		Pattern p = Pattern.compile(sentenceDelimiter);
		Matcher matcher = p.matcher(text);
		String currentPhrase;
		int currentIndex = 0;
		while(matcher.find()) {
			currentPhrase = text.substring(currentIndex, matcher.end());
			currentIndex = matcher.end();
			currentPhrase = postprocessText(currentPhrase);
			result.add(currentPhrase);
		}
    	return result;
    }
    
    public static void main(String[] args) {
		try {			
			if(args.length < 1) {
				System.out.println("Usage: java PhraseDBFiller <file-name|directory-name>");
				return;
			}
			PhraseDBFiller filler;
			try {
				filler = new PhraseDBFiller();
			} catch(Exception e) {
				System.out.println("Cannot find database properties file");
				
				e.printStackTrace();
				return;
			}
			/*filler.checkDatabaseConsistency();*/
			File file = new File(args[0]);
			if(!file.exists()) {
				System.out.println("Can't find file " + args[0]);
				return;				
			}
			
	    	try {   	
	    		RandomAccessFile randomAccess = new RandomAccessFile(file,"r");
	    		byte[] array = new byte[(int)file.length()];

	    		randomAccess.read(array);
	    		
	    		String content = new String(array); 
	    		System.out.print("Adding file \"" + file.getName() + "\"... ");
	    		Vector<String> phrases = filler.getPhrases(content);
	    		System.out.println("added (" + phrases.size() + " phrases)");
	    		File outputFile = new File("logs.txt");
                FileOutputStream out = new FileOutputStream(outputFile, true);
                PrintStream p = new PrintStream(out);
	    		p = new PrintStream(outputFile);
	    		p.println(":adding file: " + args[0]);
	    		for(int i = 0; i < phrases.size(); i++) {
	    			p.println(phrases.elementAt(i));
	    		}
	    		p.close();
	    		out.close();
	    		randomAccess.close();	    		
	    		
	    	} catch (Exception e) {
	    		e.printStackTrace();
	    	}		
	    	
			//int phrasesCount = 0;
			//if(file.isDirectory()) {
			//	phrasesCount = filler.insertDirectoryToDB(file);
			//} else {
			//	phrasesCount = filler.insertFileToDB(file);
			//}
			//System.out.println(phrasesCount + " phrases total added to DB");
		} catch(Exception e) {
			e.printStackTrace();
		}
		
	}
}
